{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "hw7_toturial.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "42d275597f7e4ed3a16e511144c72a83": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9c20f4e3786e4f0d9e3f6e8ac957aa84",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ec54e566abc741afa842e0e3a3581327",
              "IPY_MODEL_11506e8715bb49e4a40e964125814e61"
            ]
          }
        },
        "9c20f4e3786e4f0d9e3f6e8ac957aa84": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ec54e566abc741afa842e0e3a3581327": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f6fa316d9f9e4c6eabcf542d25736733",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 624,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 624,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a0a91345abb6446ab39833c6b848a140"
          }
        },
        "11506e8715bb49e4a40e964125814e61": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_87bbd44faa9042b1b55cc5dfba0e694e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 624/624 [00:13&lt;00:00, 44.7B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0bd4efad79374d10b63fa5b40cc5d73c"
          }
        },
        "f6fa316d9f9e4c6eabcf542d25736733": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a0a91345abb6446ab39833c6b848a140": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "87bbd44faa9042b1b55cc5dfba0e694e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0bd4efad79374d10b63fa5b40cc5d73c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f2228678b6ee4b45b3c0e5d06d7819af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6a2e962ee2954c0e85ed5ff612f57600",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1235625c152b43e1adc23a6462373193",
              "IPY_MODEL_b4fd2b6f2986428e9726933dc0e6a1a6"
            ]
          }
        },
        "6a2e962ee2954c0e85ed5ff612f57600": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1235625c152b43e1adc23a6462373193": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8fd5113f2b98445b9318f9f65c208c31",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 411577189,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 411577189,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c287374ce46246d8b18c82eba9a24e10"
          }
        },
        "b4fd2b6f2986428e9726933dc0e6a1a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a9cfabc73eda44eab01dbf099db31cf8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 412M/412M [00:13&lt;00:00, 30.7MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_568abdc0303a480abdf51c40cc7a455e"
          }
        },
        "8fd5113f2b98445b9318f9f65c208c31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c287374ce46246d8b18c82eba9a24e10": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a9cfabc73eda44eab01dbf099db31cf8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "568abdc0303a480abdf51c40cc7a455e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c955118e7c894124813357473d423fd8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b9d44811690844469c326cc83b4c59a0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_973a74e93552467e95ca2ff6b13bd33f",
              "IPY_MODEL_4713240943ed4a368c11a7784dda3178"
            ]
          }
        },
        "b9d44811690844469c326cc83b4c59a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "973a74e93552467e95ca2ff6b13bd33f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1c6c1f5a1b744e07a48cfe8325c75af6",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 109540,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 109540,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d95952d558eb45999e36b67876a4fe4a"
          }
        },
        "4713240943ed4a368c11a7784dda3178": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_54a87d876f60451e9fad76b5e12a50c1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 110k/110k [02:13&lt;00:00, 818B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3190ba9e729140498257b2f023589f36"
          }
        },
        "1c6c1f5a1b744e07a48cfe8325c75af6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d95952d558eb45999e36b67876a4fe4a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "54a87d876f60451e9fad76b5e12a50c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3190ba9e729140498257b2f023589f36": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f547663bfbea4ca38079792244b19de8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b03ad3c8b0ce49a5b889f7d8303f925e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6b21fb24e9c044daaec817c3d43d6305",
              "IPY_MODEL_001057bd898b4d958da9763467792473"
            ]
          }
        },
        "b03ad3c8b0ce49a5b889f7d8303f925e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6b21fb24e9c044daaec817c3d43d6305": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_30530783a32c450283c08c9909170858",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 268943,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 268943,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ebaedf860ec0428890c97628d469ea0c"
          }
        },
        "001057bd898b4d958da9763467792473": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5b38aef2f44243238cee4a696b771181",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 269k/269k [00:01&lt;00:00, 139kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4b98b18637fe4a029f62c26f8af2a460"
          }
        },
        "30530783a32c450283c08c9909170858": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ebaedf860ec0428890c97628d469ea0c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5b38aef2f44243238cee4a696b771181": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4b98b18637fe4a029f62c26f8af2a460": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8f7cc955629240ffbac4d91534e29e02": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ca8af220bcec4ba59c1a1499edaf3059",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_63425584d8ff4ce69445d10a24157dd1",
              "IPY_MODEL_51f94fab9508463d81c8a94fee1a0004"
            ]
          }
        },
        "ca8af220bcec4ba59c1a1499edaf3059": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "63425584d8ff4ce69445d10a24157dd1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6981e55504d2408981c32ac88c142eaa",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 29,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 29,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_25a4a4b87e3d443ba57550b0746d984a"
          }
        },
        "51f94fab9508463d81c8a94fee1a0004": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_28159f9119cd47348c43880e1dfe3dab",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 29.0/29.0 [00:00&lt;00:00, 44.9B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b90b8c20a2904fa79ce0d3d889074b66"
          }
        },
        "6981e55504d2408981c32ac88c142eaa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "25a4a4b87e3d443ba57550b0746d984a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "28159f9119cd47348c43880e1dfe3dab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b90b8c20a2904fa79ce0d3d889074b66": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b1a742f5cd604a29b29ba41fe4a1e1ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d4cfa3eb975a40888ec232783606dc63",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_392b126320634fa0a54d1e4d926a52de",
              "IPY_MODEL_55885ffb849541499b22b36d264b7030"
            ]
          }
        },
        "d4cfa3eb975a40888ec232783606dc63": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "392b126320634fa0a54d1e4d926a52de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_32efba59d83741c883764b353abe1b94",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 213450,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 213450,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2e4a80defa004b89b0e984517e110d2c"
          }
        },
        "55885ffb849541499b22b36d264b7030": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_564c00bd637446aaa1ef2dc36b727583",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 213k/213k [02:11&lt;00:00, 1.62kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e1fdd3b7534f4048ba5f0dc5e0de686b"
          }
        },
        "32efba59d83741c883764b353abe1b94": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2e4a80defa004b89b0e984517e110d2c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "564c00bd637446aaa1ef2dc36b727583": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e1fdd3b7534f4048ba5f0dc5e0de686b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fe6157bd422f4936af9ccb22f3065936": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9765c633a062488cb63bbf2b562e6d4b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9594e45d6669482d83a2d4fe1acc8ce4",
              "IPY_MODEL_68495d26b19b450284039d33078e7921"
            ]
          }
        },
        "9765c633a062488cb63bbf2b562e6d4b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9594e45d6669482d83a2d4fe1acc8ce4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7d4bdd11ea3f4356b0343dfa2aab21cb",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 435797,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 435797,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f3381db964fe412e8c1865de87484810"
          }
        },
        "68495d26b19b450284039d33078e7921": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d9f7b71123164721bd94b290388e6fae",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 436k/436k [00:50&lt;00:00, 8.59kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_362628a10dc44fbd81b078157c9e4b0f"
          }
        },
        "7d4bdd11ea3f4356b0343dfa2aab21cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f3381db964fe412e8c1865de87484810": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d9f7b71123164721bd94b290388e6fae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "362628a10dc44fbd81b078157c9e4b0f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f774866e47ef416a960555b3d6c36aed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1c77977c95434d4092e3c9b0fead2973",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a26c669b5d5d4b59b8268c5c2922b836",
              "IPY_MODEL_b6845eb5ea6942c490eb31136efc3d62"
            ]
          }
        },
        "1c77977c95434d4092e3c9b0fead2973": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a26c669b5d5d4b59b8268c5c2922b836": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_31575a3f35d743779c93e09192af8fcf",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 29,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 29,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cf018707ae4e44779cc388108f996705"
          }
        },
        "b6845eb5ea6942c490eb31136efc3d62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c8b5f28e0df84f5f8ef9b664b6c74afb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 29.0/29.0 [00:49&lt;00:00, 1.70s/B]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_714453927123402587cb475df530f331"
          }
        },
        "31575a3f35d743779c93e09192af8fcf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cf018707ae4e44779cc388108f996705": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c8b5f28e0df84f5f8ef9b664b6c74afb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "714453927123402587cb475df530f331": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cHB4GJNQISqj"
      },
      "source": [
        "Install transformers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AhpM9uzIIJVT",
        "outputId": "ce534bf9-5bea-4512-c0ee-5771fdc7b787"
      },
      "source": [
        "!pip install transformers==4.5.0"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers==4.5.0\n",
            "  Downloading transformers-4.5.0-py3-none-any.whl (2.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.1 MB 9.1 MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 50.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==4.5.0) (3.0.12)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.5.0) (1.19.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==4.5.0) (2.23.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers==4.5.0) (4.6.1)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.45-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 73.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers==4.5.0) (21.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==4.5.0) (4.41.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.5.0) (2019.12.20)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers==4.5.0) (3.5.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers==4.5.0) (3.7.4.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers==4.5.0) (2.4.7)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.5.0) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.5.0) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.5.0) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.5.0) (2021.5.30)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.5.0) (1.0.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.5.0) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.5.0) (7.1.2)\n",
            "Installing collected packages: tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.45 tokenizers-0.10.3 transformers-4.5.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "odDp0WQqJTI-"
      },
      "source": [
        "Import Packages"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n3Q1zyaqJVNO"
      },
      "source": [
        "import torch\n",
        "from transformers import AdamW, BertForQuestionAnswering, BertTokenizerFast"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cdr-kb89JizL"
      },
      "source": [
        "load model and tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 227,
          "referenced_widgets": [
            "42d275597f7e4ed3a16e511144c72a83",
            "9c20f4e3786e4f0d9e3f6e8ac957aa84",
            "ec54e566abc741afa842e0e3a3581327",
            "11506e8715bb49e4a40e964125814e61",
            "f6fa316d9f9e4c6eabcf542d25736733",
            "a0a91345abb6446ab39833c6b848a140",
            "87bbd44faa9042b1b55cc5dfba0e694e",
            "0bd4efad79374d10b63fa5b40cc5d73c",
            "f2228678b6ee4b45b3c0e5d06d7819af",
            "6a2e962ee2954c0e85ed5ff612f57600",
            "1235625c152b43e1adc23a6462373193",
            "b4fd2b6f2986428e9726933dc0e6a1a6",
            "8fd5113f2b98445b9318f9f65c208c31",
            "c287374ce46246d8b18c82eba9a24e10",
            "a9cfabc73eda44eab01dbf099db31cf8",
            "568abdc0303a480abdf51c40cc7a455e"
          ]
        },
        "id": "QAy8IH_7Jmuy",
        "outputId": "4db2ff4b-7e47-4a5c-bfa1-7b22826cffe1"
      },
      "source": [
        "model_name = \"bert-base-chinese\"\n",
        "model = BertForQuestionAnswering.from_pretrained(model_name)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "42d275597f7e4ed3a16e511144c72a83",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=624.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f2228678b6ee4b45b3c0e5d06d7819af",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=411577189.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-chinese were not used when initializing BertForQuestionAnswering: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForQuestionAnswering were not initialized from the model checkpoint at bert-base-chinese and are newly initialized: ['qa_outputs.weight', 'qa_outputs.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 317,
          "referenced_widgets": [
            "c955118e7c894124813357473d423fd8",
            "b9d44811690844469c326cc83b4c59a0",
            "973a74e93552467e95ca2ff6b13bd33f",
            "4713240943ed4a368c11a7784dda3178",
            "1c6c1f5a1b744e07a48cfe8325c75af6",
            "d95952d558eb45999e36b67876a4fe4a",
            "54a87d876f60451e9fad76b5e12a50c1",
            "3190ba9e729140498257b2f023589f36",
            "f547663bfbea4ca38079792244b19de8",
            "b03ad3c8b0ce49a5b889f7d8303f925e",
            "6b21fb24e9c044daaec817c3d43d6305",
            "001057bd898b4d958da9763467792473",
            "30530783a32c450283c08c9909170858",
            "ebaedf860ec0428890c97628d469ea0c",
            "5b38aef2f44243238cee4a696b771181",
            "4b98b18637fe4a029f62c26f8af2a460",
            "8f7cc955629240ffbac4d91534e29e02",
            "ca8af220bcec4ba59c1a1499edaf3059",
            "63425584d8ff4ce69445d10a24157dd1",
            "51f94fab9508463d81c8a94fee1a0004",
            "6981e55504d2408981c32ac88c142eaa",
            "25a4a4b87e3d443ba57550b0746d984a",
            "28159f9119cd47348c43880e1dfe3dab",
            "b90b8c20a2904fa79ce0d3d889074b66",
            "b1a742f5cd604a29b29ba41fe4a1e1ae",
            "d4cfa3eb975a40888ec232783606dc63",
            "392b126320634fa0a54d1e4d926a52de",
            "55885ffb849541499b22b36d264b7030",
            "32efba59d83741c883764b353abe1b94",
            "2e4a80defa004b89b0e984517e110d2c",
            "564c00bd637446aaa1ef2dc36b727583",
            "e1fdd3b7534f4048ba5f0dc5e0de686b",
            "fe6157bd422f4936af9ccb22f3065936",
            "9765c633a062488cb63bbf2b562e6d4b",
            "9594e45d6669482d83a2d4fe1acc8ce4",
            "68495d26b19b450284039d33078e7921",
            "7d4bdd11ea3f4356b0343dfa2aab21cb",
            "f3381db964fe412e8c1865de87484810",
            "d9f7b71123164721bd94b290388e6fae",
            "362628a10dc44fbd81b078157c9e4b0f",
            "f774866e47ef416a960555b3d6c36aed",
            "1c77977c95434d4092e3c9b0fead2973",
            "a26c669b5d5d4b59b8268c5c2922b836",
            "b6845eb5ea6942c490eb31136efc3d62",
            "31575a3f35d743779c93e09192af8fcf",
            "cf018707ae4e44779cc388108f996705",
            "c8b5f28e0df84f5f8ef9b664b6c74afb",
            "714453927123402587cb475df530f331"
          ]
        },
        "id": "bcGIq-YdIgbv",
        "outputId": "e84a98e8-06de-406c-fe74-d3c37e2a6b48"
      },
      "source": [
        "chi_tokenizer = BertTokenizerFast.from_pretrained(\"bert-base-chinese\")\n",
        "eng_tokenizer = BertTokenizerFast.from_pretrained(\"bert-base-cased\")"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c955118e7c894124813357473d423fd8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=109540.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f547663bfbea4ca38079792244b19de8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=268943.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8f7cc955629240ffbac4d91534e29e02",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=29.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b1a742f5cd604a29b29ba41fe4a1e1ae",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=213450.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fe6157bd422f4936af9ccb22f3065936",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=435797.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f774866e47ef416a960555b3d6c36aed",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=29.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v47JVEBMKWmg"
      },
      "source": [
        "Tokenize"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 245
        },
        "id": "tVpXlYFqKfvF",
        "outputId": "2e024b4d-bf51-46d5-ee1f-70e807dbbaa7"
      },
      "source": [
        "chi_paragraph = \"李宏毅几班大金。2021 ML\"\n",
        "tokens = chi_tokenizer.tokenize(chi_paragraph)\n",
        "print(tokens)\n",
        "chi_tokenizer.convert_tokens_to_ids(tokens)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-6-5345b3db793d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mchi_paragraph\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"李宏毅几班大金。2021 ML\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mtokens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mchi_tokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchi_paragraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0manswer_start_token\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtokens\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchar_to_token\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m12\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0manswer_end_token\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtokens\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchar_to_token\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m14\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'list' object has no attribute 'char_to_token'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7eYYO7OaLEqq",
        "outputId": "a1f9ef2c-b63b-4cfb-e1cc-cdba9314443a"
      },
      "source": [
        "eng_paragraph = \"Lee Hung-yi which class DaiKin\"\n",
        "tokens = eng_tokenizer.tokenize(eng_paragraph)\n",
        "print(tokens)\n",
        "eng_tokenizer.convert_tokens_to_ids(tokens)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['Lee', 'Hung', '-', 'y', '##i', 'which', 'class', 'Dai', '##K', '##in']\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[2499, 26157, 118, 194, 1182, 1134, 1705, 23084, 2428, 1394]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "srG5806YLrY7"
      },
      "source": [
        "Encoder vs Decoder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rfkLsWEQLuq6",
        "outputId": "5dbd05df-c9aa-4667-c19d-9ae529b4f5e9"
      },
      "source": [
        "question = \"李宏毅几班\"\n",
        "paragraph = \"李宏毅几班大金\"\n",
        "encoded = chi_tokenizer.encode(question,paragraph)\n",
        "decoded = chi_tokenizer.decode(encoded)\n",
        "print(encoded,decoded)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[101, 3330, 2131, 3675, 1126, 4408, 102, 3330, 2131, 3675, 1126, 4408, 1920, 7032, 102] [CLS] 李 宏 毅 几 班 [SEP] 李 宏 毅 几 班 大 金 [SEP]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H4-fZzE2MMJr"
      },
      "source": [
        "Model Input"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_yyq_EskMPFK",
        "outputId": "87f3fda0-3272-441d-e4f1-cf16d0304180"
      },
      "source": [
        "inputs = chi_tokenizer(question, paragraph, return_tensors='pt')\n",
        "print('Input ids:',inputs[\"input_ids\"])\n",
        "print('Token type ids: ',inputs['token_type_ids'])\n",
        "print(\"Attention mask: \",inputs['attention_mask'])"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input ids: tensor([[ 101, 3330, 2131, 3675, 1126, 4408,  102, 3330, 2131, 3675, 1126, 4408,\n",
            "         1920, 7032,  102]])\n",
            "Token type ids:  tensor([[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]])\n",
            "Attention mask:  tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4QAJGsdmR5mT"
      },
      "source": [
        "Testing(Chinese)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ppLlkYAYR_EP",
        "outputId": "8be57441-52a4-49ce-dd25-86f406dc3161"
      },
      "source": [
        "question = '李宏毅几班'\n",
        "paragraph = '李宏毅几班大金'\n",
        "inputs = chi_tokenizer(question, paragraph, return_tensors='pt')\n",
        "print(\"********\")\n",
        "print(inputs)\n",
        "\n",
        "with torch.no_grad():\n",
        "  output = model(**inputs)\n",
        "\n",
        "print('start_logits: ')\n",
        "print(output.start_logits)\n",
        "\n",
        "print(\"end_logits: \")\n",
        "print(output.end_logits)\n",
        "\n",
        "start = torch.argmax(output.start_logits)\n",
        "end = torch.argmax(output.end_logits)\n",
        "print(\"start position: \",start.item())\n",
        "print(\"end position: \",end.item())\n",
        "\n",
        "predict_id = inputs[\"input_ids\"][0][start:end+1]\n",
        "print(\"predict_id: \",predict_id)\n",
        "\n",
        "predict_answer = chi_tokenizer.decode(predict_id)\n",
        "print(\"predict_answer: \",predict_answer)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "********\n",
            "{'input_ids': tensor([[ 101, 3330, 2131, 3675, 1126, 4408,  102, 3330, 2131, 3675, 1126, 4408,\n",
            "         1920, 7032,  102]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n",
            "start_logits: \n",
            "tensor([[-1.0027, -1.7406, -1.7981, -1.7428, -1.2767, -1.1733, -1.0885, -1.5355,\n",
            "         -1.8211, -1.8030, -1.2616, -1.2871, -0.8432,  0.8675, -1.0885]])\n",
            "end_logits: \n",
            "tensor([[ 0.6023, -0.3088, -0.6113, -0.6836, -0.8636, -0.3114,  1.8552, -0.2954,\n",
            "         -0.6125, -0.7852, -0.8407, -0.4104, -0.2830, -0.0199,  1.8552]])\n",
            "start position:  13\n",
            "end position:  14\n",
            "predict_id:  tensor([7032,  102])\n",
            "predict_answer:  金 [SEP]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ljHrg9yyUHMq"
      },
      "source": [
        "Train(Chinese)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Tuqrq0ULULNu",
        "outputId": "cbc86713-dbc1-45d1-b6bb-45d2a50c5e27"
      },
      "source": [
        "output = model(**inputs, start_positions=torch.tensor([13]), end_positions=torch.tensor([14]))\n",
        "print(\"loss: \",output.loss)\n",
        "\n",
        "optimizer = AdamW(model.parameters(), lr=1e-4)\n",
        "output.loss.backward()\n",
        "optimizer.step()"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loss:  tensor(2.8654, grad_fn=<DivBackward0>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NGdrfFT1YD5e"
      },
      "source": [
        "Testing(English)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Nbo5_ktiYImB",
        "outputId": "44de26fc-f3de-4bdc-e9f9-1ae03be7bdf0"
      },
      "source": [
        "question = \"why does Jeanie like Tom?\"\n",
        "paragraph = \"Jeanie likes Tom because Tom is good at deep learning\"\n",
        "inputs = eng_tokenizer(question, paragraph, return_tensors='pt')\n",
        "print(\"********\")\n",
        "print(inputs)\n",
        "print(\"*********\")\n",
        "\n",
        "with torch.no_grad():\n",
        "  output = model(**inputs)\n",
        "\n",
        "print(\"start_logits: \",output.start_logits)\n",
        "print(\"end_logits: \",output.end_logits)\n",
        "\n",
        "start = torch.argmax(output.start_logits)\n",
        "end = torch.argmax(output.end_logits)\n",
        "print(\"start_position: \",start.item())\n",
        "print(\"end_position: \",end.item())\n",
        "\n",
        "predict_id = inputs['input_ids'][0][start:end+1]\n",
        "print(\"predict_id: \",predict_id)\n",
        "\n",
        "predict_answer = eng_tokenizer.decode(predict_id)\n",
        "print(\"predict_answer: \",predict_answer)"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "********\n",
            "{'input_ids': tensor([[ 101, 1725, 1674, 2893, 1663, 1176, 2545,  136,  102, 2893, 1663, 7407,\n",
            "         2545, 1272, 2545, 1110, 1363, 1120, 1996, 3776,  102]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n",
            "*********\n",
            "start_logits:  tensor([[-0.5179, -0.6189, -0.0827, -0.9092, -0.5319, -0.6465, -0.0302, -0.0608,\n",
            "         -0.9533, -0.8208, -0.3877, -0.4088, -0.3412, -0.2008,  0.2864,  0.0029,\n",
            "         -0.0646, -0.1460,  0.0119,  0.1088, -0.9533]])\n",
            "end_logits:  tensor([[ 0.1546, -0.4161,  0.2289, -0.5821, -0.3508, -0.3789, -0.1332,  0.1198,\n",
            "          0.8450, -0.5445, -0.1598, -0.2418, -0.1378, -0.0047, -0.0694, -0.0811,\n",
            "          0.1694, -0.1028,  0.1601,  1.0741,  0.8450]])\n",
            "start_position:  14\n",
            "end_position:  19\n",
            "predict_id:  tensor([2545, 1110, 1363, 1120, 1996, 3776])\n",
            "predict_answer:  Tom is good at deep learning\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xBUF7zYwaMSX"
      },
      "source": [
        "Train(English)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kyTi8h3TaK7p",
        "outputId": "e20336f3-b675-415b-c219-68aa475d3c5f"
      },
      "source": [
        "output = model(**inputs, start_positions=torch.tensor([14]), end_positions=torch.tensor([19]))\n",
        "print(\"loss: \",output.loss)\n",
        "\n",
        "optimizer = AdamW(model.parameters(), lr=1e-4)\n",
        "output.loss.backward()\n",
        "optimizer.step()"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loss:  tensor(3.2611, grad_fn=<DivBackward0>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HuHaULCUVQsw"
      },
      "source": [
        "Learning rate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v6d6s_xIVQMZ",
        "outputId": "1e8b6093-8cc3-4aac-a3d0-f8bbc0551d0d"
      },
      "source": [
        "learning_rate = 1e-4\n",
        "optimizer = AdamW(model.parameters(),lr=learning_rate)\n",
        "total_step = 1000\n",
        "lr = []\n",
        "for i in range(total_step):\n",
        "  optimizer.param_groups[0]['lr'] -= learning_rate / total_step\n",
        "  lr.append(optimizer.param_groups[0]['lr'])\n",
        "\n",
        "print(optimizer.param_groups[0]['lr'])\n"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-1.5863074217500927e-18\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "hyEufom8ZZz5",
        "outputId": "e701729b-0669-45ec-a77a-93e484647621"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "x = list(range(0,1000))\n",
        "y = lr\n",
        "plt.plot(x,y)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7fbc2d06a050>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD4CAYAAAAQP7oXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3hUddr/8fedhIQeWkCkhSoGRMFIJ3GVroIFFWyoKBaQEncVnm3q+ttdtiSigoJiL4BYiIo0YRNAWhDpxdBBhNAFpH9/f+T4PNmYkCEkmczk87ourp35nnPuuQ+H9cPMGe6Ycw4RERFfhPi7ARERCRwKDRER8ZlCQ0REfKbQEBERnyk0RETEZ2H+bqAwVatWzUVHR/u7DRGRgLJs2bJ9zrmonLYFdWhER0eTlpbm7zZERAKKmW3LbZs+nhIREZ8pNERExGcKDRER8ZlCQ0REfKbQEBERn/kUGmbW3cw2mFm6mY3IYXuEmU3yti82s+gs20Z66xvMrFteNc1ssLfmzKxalnUzsxe9bSvNrFV+T1pERPInz9Aws1BgDNADiAH6mVlMtt0GAAedc42AJGCUd2wM0BdoBnQHxppZaB41FwCdgexf+eoBNPZ+DQReubBTFRGRi+XLO43WQLpzbrNz7hQwEeidbZ/ewNve4ynA9WZm3vpE59xJ59wWIN2rl2tN59xy59zWHProDbzjMi0CKplZzQs5WV9tyjjKv2du4MTps4VRXkQkYPkSGrWAHVme7/TWctzHOXcGOAxUPc+xvtTMTx+Y2UAzSzOztIyMjDxK5mzW2j28NCedG16cx7JtB/JVQ0QkGAXdjXDn3HjnXKxzLjYqKsd/BZ+nR+Mb8vaDrTlx+hx9Xl3IM8lrOHbyTAF3KiISeHwJjV1AnSzPa3trOe5jZmFAJLD/PMf6UjM/fRSY+CZRzBgex31t6/H2wq10TUoldWP+3rmIiAQLX0JjKdDYzOqbWTiZN7aTs+2TDPT3HvcB5rjMnyObDPT1vl1Vn8yb2Et8rJldMnCf9y2qtsBh59xuH/rPt/IRYTzbuzmTH2lHRKkQ7ntjCb/9aAWHjp8qzJcVESm28gwN7x7FYGAGsA6Y7JxbY2bPmVkvb7cJQFUzSwcSgBHesWuAycBaYDowyDl3NreaAGY2xMx2kvlOYqWZve69xjRgM5k3018DHr/os/fRNdFVmDakE49f25BPl++ic2IqX60q1LwSESmWLPMNQXCKjY11BT3ldvWuwzw1ZSVrdx+hR/NLeLZ3M6pXKF2gryEi4k9mtsw5F5vTtqC7EV7YmteKZOrgDjzV/TK+Xr+XLompfJS2g2AOXxGRXyg08qFUaAiPX9uIr4Z2okmN8vxuykrue2MJOw4c93drIiKFSqFxERpGlWfSwHY817sZ3247SLcXUnlrwRbOndO7DhEJTgqNixQSYtzXLpoZw+OIja7CM5+v5fZxC0nf+5O/WxMRKXAKjQJSu3JZ3n7gGv59+5Wk7z1Kz9HzGTM3ndNnz/m7NRGRAqPQKEBmxm1X12Z2QjydY6rzzxkb6P3yAlbvOuzv1kRECoRCoxBEVYhg7N1X8+o9V5Nx9CS9xyxg1PT1GoAoIgFPoVGIuje/hNnD47mtVS1e+c8meo6ex5ItGoAoIoFLoVHIIsuW4h99ruS9AW04dfYcd4xbyB8/W81RDUAUkQCk0CgiHRtXY8awOB7oEM17i7fRNTGFuRv2+rstEZELotAoQuUiwvjzTc2Y8mh7ykaE8cCbS0mY9B0Hj2kAoogEBoWGH1xdrzJfDunIE9c1InnFD3RJSuHLlbs1ikREij2Fhp9EhIXyZNfLSB7ckZqRZRj0wbc88u4y9h454e/WRERypdDws5hLK/Lp4+0Z2aMpKRszuD4xhclLNQBRRIonhUYxEBYawiPxDflqaCcur1mRpz5eyT0TFrN9vwYgikjxotAoRhpElWfiw215/ubmrNhxmG4vpDJh/hbOagCiiBQTCo1iJiTEuKdtPWYOj6NNgyr85Yu19Hn1G77fowGIIuJ/Co1i6tJKZXjz/mt44c6r2LrvGDe8OJ8Xv/6eU2c0AFFE/EehUYyZGTe3rMWshHi6Nb+ExFkb6fXyfFbuPOTv1kSkhFJoBIBq5SN4qV9LXrsvloPHT3HzmAX8bdo6fj6lAYgiUrQUGgGkS0wNZg6P585r6jAudTM9RqeyaPN+f7clIiWIQiPARJYpxd9ubcEHD7XhnIO+4xfx+09X8dOJ0/5uTURKAIVGgGrfqBrTh3XioY71+XDJdrompTJn/R5/tyUiQU6hEcDKhofxhxtj+Pix9lQoHcaDb6UxbOJyDmgAoogUEoVGEGhZtzJfPNGJodc35stVu+mcmELyih80ikRECpxCI0iEh4UwvEsTPn+iI3Uql2HIh8t5+J1l/HhYAxBFpOAoNIJM00sq8snjHfh9z8uZn55Bl8QUPlyyXe86RKRAKDSCUGiI8XBcA6YPjaNZrYqM/GQVd722mG37j/m7NREJcAqNIBZdrRwfPNSWv916Bat3ZQ5AfH3eZg1AFJF8U2gEuZAQo1/rusxKiKdjo2o8/+U6bn3lGzb8qAGIInLhfAoNM+tuZhvMLN3MRuSwPcLMJnnbF5tZdJZtI731DWbWLa+aZlbfq5Hu1Qz31uua2VwzW25mK82s58WceElzSWRpXrsvlhf7tWTHgePc+NI8kmZt1ABEEbkgeYaGmYUCY4AeQAzQz8xisu02ADjonGsEJAGjvGNjgL5AM6A7MNbMQvOoOQpI8mod9GoD/AGY7Jxr6dUcm79TLrnMjF5XXsrshHh6XlGT0V9/z40vzeO7HRqAKCK+8eWdRmsg3Tm32Tl3CpgI9M62T2/gbe/xFOB6MzNvfaJz7qRzbguQ7tXLsaZ3zHVeDbyaN3uPHVDRexwJ/HBhpyq/qFIunNF9WzKhfyxHfj7DrWMX8PwXazUAUUTy5Eto1AJ2ZHm+01vLcR/n3BngMFD1PMfmtl4VOOTVyP5azwD3mNlOYBrwhA+9y3lcf3kNZibE0bd1XV6fv4VuL6TyzaZ9/m5LRIqxQLoR3g94yzlXG+gJvGtmv+rfzAaaWZqZpWVkZBR5k4GmYulS/PWWK/jw4baEGNz12mJGfrKSIxqAKCI58CU0dgF1sjyv7a3luI+ZhZH58dH+8xyb2/p+oJJXI/trDQAmAzjnFgKlgWrZm3XOjXfOxTrnYqOionw4PQFo17AqXw2N45G4BkxauoMuiSnMXqsBiCLy33wJjaVAY+9bTeFk3oROzrZPMtDfe9wHmOMy/wlyMtDX+3ZVfaAxsCS3mt4xc70aeDWneo+3A9cDmNnlZIaG3koUoDLhoYzseTmfDepA5bLhPPROGk98uJx9R0/6uzURKSbyDA3v/sJgYAawjsxvMK0xs+fMrJe32wSgqpmlAwnACO/YNWS+O1gLTAcGOefO5lbTq/U0kODVqurVBngSeNjMVgAfAvc7zcYoFC1qVyJ5cEcSujRh+urddElM4bPluzSKRESwYP4PQWxsrEtLS/N3GwFt456feGrKSr7bcYjrmlbn+Zubc2mlMv5uS0QKkZktc87F5rQtkG6Eix80qVGBjx9rzx9vjGHhpv10TUrlvUXbOKdRJCIlkkJD8hQaYgzoWJ8Zw+K4sk4kf/hsNf1eW8SWfRqAKFLSKDTEZ3WrluW9AW34x20tWLv7CN1fSGVcyibOnNUoEpGSQqEhF8TMuOOaOsxOiCeuSRR/+2o9t4z9hrU/HPF3ayJSBBQaki81KpZm/L1XM+auVuw+/DO9Xp7Pv2du4OQZjSIRCWYKDck3M+OGFjWZNTyeXldeyktz0rnhxfks23bQ362JSCFRaMhFq1wunMQ7r+LNB67h+Mkz9Hn1G579fA3HT53J+2ARCSgKDSkwv7msOjMT4rm3bT3eXLCVrkmpzP9eAxBFgolCQwpU+YgwnuvdnMmPtKNUaAj3TFjMU1NWcPhnDUAUCQYKDSkUretX4auhnXjs2oZ8/O0uuiSmMGPNj/5uS0QukkJDCk3pUqE83b0pnz3egarlI3jk3WUMev9bMn7SAESRQKXQkEJ3Re1Ikgd34HfdLmPW2j10Tkzh42U7NQBRJAApNKRIlAoNYdBvGjFtaEcaVS/Pkx+t4P43l7Lr0M/+bk1ELoBCQ4pUo+oV+OiRdjxzUwxLtx6ga2IK7yzcqgGIIgFCoSFFLiTEuL9D5gDEVvUq86epa7hz/EI2ZRz1d2sikgeFhvhNnSpleefB1vyzTws2/PgTPUbPY+x/0jmtAYgixZZCQ/zKzLg9tg6zn4znusuq84/pG7h5zAJW7zrs79ZEJAcKDSkWqlcozav3Xs0rd7diz5GT9B6zgH/OWM+J0xqAKFKcKDSkWOlxRU1mJ8RxS8tajJm7iZ4vziNt6wF/tyUiHoWGFDuVyobzr9uv5J0HW3Py9DluH7eQZ5LXcOykBiCK+JtCQ4qtuCZRzBweR/920by9MHMAYsrGDH+3JVKiKTSkWCsXEcYzvZrx0SPtiCgVQv83lvDk5BUcOn7K362JlEgKDQkIsdFVmDakE4N+05DPvttF58RUvlq1299tiZQ4Cg0JGKVLhfK7bk1JHtyBGhUjeOz9b3n03WXsPXLC362JlBgKDQk4zS6NZOqgDjzdvSlzNuylc2IKH6Xt0ABEkSKg0JCAFBYawmPXNuSroZ247JIK/G7KSu57Ywk7Dhz3d2siQU2hIQGtYVR5Jg1sx196N+PbbQfp9kIqby3YwlkNQBQpFAoNCXghIca97aKZMTyOa6Kr8Mzna7lj3ELS9/7k79ZEgo5CQ4JG7cpleeuBa0i840o2ZRyl5+j5vDznew1AFClACg0JKmbGra1qM2t4PF2a1eBfMzfS62UNQBQpKAoNCUpRFSIYc1crxt17NfuOZg5A/PtXGoAocrF8Cg0z625mG8ws3cxG5LA9wswmedsXm1l0lm0jvfUNZtYtr5pmVt+rke7VDM+y7Q4zW2tma8zsg/yetJQc3Zpdwuzh8fRpVZtXUzbRc/Q8lmzRAESR/MozNMwsFBgD9ABigH5mFpNttwHAQedcIyAJGOUdGwP0BZoB3YGxZhaaR81RQJJX66BXGzNrDIwEOjjnmgHD8n3WUqJEli3FqD4teG9AG06dPccd4xbyx89W89OJ0/5uTSTg+PJOozWQ7pzb7Jw7BUwEemfbpzfwtvd4CnC9mZm3PtE5d9I5twVI9+rlWNM75jqvBl7Nm73HDwNjnHMHAZxzey/8dKUk69i4GjOHx/Fgh/q8t3gb3ZJSmbtBf4xELoQvoVEL2JHl+U5vLcd9nHNngMNA1fMcm9t6VeCQVyP7azUBmpjZAjNbZGbdc2rWzAaaWZqZpWVkaCKq/Ley4WH86aYYpjzannIRYTzw5lISJn3HwWMagCjii0C6ER4GNAauBfoBr5lZpew7OefGO+dinXOxUVFRRdyiBIqr61XmiyEdGXJdI5JX/EDnxBS+WPmDRpGI5MGX0NgF1MnyvLa3luM+ZhYGRAL7z3Nsbuv7gUpejeyvtRNIds6d9j7q2khmiIjkS0RYKAldL+PzJzpyaaUyDP5gOY+8u4w9GoAokitfQmMp0Nj7VlM4mTe2k7Ptkwz09x73Aea4zL+yJQN9vW9X1SfzP/JLcqvpHTPXq4FXc6r3+DMy32VgZtXI/Lhq8wWer8ivXF6zIp8+3p6RPZqSsjGDzokpTFq6Xe86RHKQZ2h49xcGAzOAdcBk59waM3vOzHp5u00AqppZOpAAjPCOXQNMBtYC04FBzrmzudX0aj0NJHi1qnq18fbdb2ZryQyW3znn9l/c6YtkCgsN4ZH4hkwfFsflNSvy9MeruPv1xWzfrwGIIllZMP9tKjY21qWlpfm7DQkw5845Ply6nb9NW8/Zc47fdruM+9tHExpi/m5NpEiY2TLnXGxO2wLpRrhIkQgJMe5uU49ZCXG0a1iVv3yxltte+YaNezQAUUShIZKLmpFlmNA/ltF9r2Lb/mPc8OI8Xvz6e06d0QBEKbkUGiLnYWb0vqoWsxPi6d68JomzNtLr5fms2HHI362J+IVCQ8QHVctH8FK/lrx2XywHj5/ilrEL+Ou0dfx8SgMQpWRRaIhcgC4xNZiVEM+d19RhfOpmeoxOZeEmfYlPSg6FhsgFqli6FH+7tQUfPNSGcw76vbaI//l0FUc0AFFKAIWGSD61b1SNGcPieLhTfSYu2U7XxFTmrN/j77ZECpVCQ+QilAkP5fc3xPDJ4x2ILFOKB99KY+jE5ew/etLfrYkUCoWGSAG4qk4lPn+iI8M6N2baqt10SUoleYUGIErwUWiIFJDwsBCGdW7CF090ok6Vsgz5cDkPv5PG7sM/+7s1kQKj0BApYJddUoFPHmvPH264nPnp++iamMoHi7dz7pzedUjgU2iIFILQEOOhTg2YMSyO5rUi+Z9PV3HX64vYuu+Yv1sTuSgKDZFCVK9qOT54uA1/v/UK1uw6QvfRqbyWupmzetchAUqhIVLIzIy+resyKyGejo2q8f+mrePWsQvY8KMGIErgUWiIFJFLIkvz2n2xvNSvJTsP/syNL80jadZGTp7RKBIJHAoNkSJkZtx05aXMSojnhitqMvrr77nppfks337Q362J+EShIeIHVcqF80Lflrxxfyw/nTjDra98w1++WMvxU2f83ZrIeSk0RPzouqY1mDk8jrvb1GXC/C10f2Ee36Tv83dbIrlSaIj4WYXSpXj+5iuYOLAtIQZ3vb6YER+v5PDPGoAoxY9CQ6SYaNugKtOHxfFIfAMmp+2ga1IKs9ZqAKIULwoNkWKkdKlQRva4nM8GdaBy2XAefieNwR98yz4NQJRiQqEhUgy1qF2J5MEdebJLE2au2UPnxBQ+Xb5TAxDF7xQaIsVUeFgIT1zfmC+HdKR+tXIMn7SCB99ayg+HNABR/EehIVLMNa5RgSmPtudPN8awaPMBuial8u6ibRqAKH6h0BAJAKEhxoMd6zNzeBxX1anEHz9bTd/XFrFFAxCliCk0RAJInSpleXdAa/5xWwvW7T5C9xdSeTVlE2fOnvN3a1JCKDREAoyZccc1dZidEE98kyj+/tV6bhn7DWt/OOLv1qQEUGiIBKgaFUsz7t6rGXt3K3Yf/pleL8/n3zM3aACiFCqFhkgAMzN6XlGTWcPj6XXVpbw0J50bXpzPsm0agCiFQ6EhEgQqlwsn8Y6reOuBa/j51Fn6vPoNz36+hmMnNQBRCpZCQySIXHtZdWYMj+PetvV4c8FWur2QyrzvM/zdlgQRn0LDzLqb2QYzSzezETlsjzCzSd72xWYWnWXbSG99g5l1y6ummdX3aqR7NcOzvdZtZubMLDY/JywS7MpHhPFc7+ZMfqQd4aEh3DthCU9NWcHh4xqAKBcvz9Aws1BgDNADiAH6mVlMtt0GAAedc42AJGCUd2wM0BdoBnQHxppZaB41RwFJXq2DXu1feqkADAUW5+90RUqO1vWrMG1oJx67tiEff7uLzkkpTF/9o7/bkgDnyzuN1kC6c26zc+4UMBHonW2f3sDb3uMpwPVmZt76ROfcSefcFiDdq5djTe+Y67waeDVvzvI6fyEzVE5c4HmKlEilS4XydPemTB3UgajyETz63jIef38Ze3/S/4Ukf3wJjVrAjizPd3prOe7jnDsDHAaqnufY3NarAoe8Gv/1WmbWCqjjnPvyfM2a2UAzSzOztIwMfZYrAtC8ViRTB3fgd90uY/a6vXRJTOXjZRqAKBcuIG6Em1kIkAg8mde+zrnxzrlY51xsVFRU4TcnEiBKhYYw6DeNmDakE42ql+fJj1bQ/82l7Dx43N+tSQDxJTR2AXWyPK/treW4j5mFAZHA/vMcm9v6fqCSVyPregWgOfAfM9sKtAWSdTNc5MI1ql6ejx5px7O9mpG29QDdklJ5Z+FWDUAUn/gSGkuBxt63msLJvLGdnG2fZKC/97gPMMdlvu9NBvp6366qDzQGluRW0ztmrlcDr+ZU59xh51w151y0cy4aWAT0cs6l5fO8RUq0kBCjf/toZgyLo1W9yvxp6hruHL+QTRlH/d2aFHN5hoZ3f2EwMANYB0x2zq0xs+fMrJe32wSgqpmlAwnACO/YNcBkYC0wHRjknDubW02v1tNAglerqldbRApBnSpleefB1vzr9ivZuOcoPUbPY8zcdE5rAKLkwoL5RlhsbKxLS9ObERFf7P3pBM8kr2Haqh9pdmlFRt3Wgua1Iv3dlviBmS1zzuX48X9A3AgXkcJXvUJpxt59Na/e04o9R07Se8wC/jF9PSdOawCi/B+Fhoj8l+7Na/J1Qjy3tqzF2P9soueL80jbesDfbUkxodAQkV+JLFuKf95+Je882JqTp89x+7iF/Hnqao5qAGKJp9AQkVzFNYli5vA4+reL5p1F2+iWlErKRv2j2ZJMoSEi51UuIoxnejVjyqPtKF0qhP5vLCFh8nccOn7K362JHyg0RMQnV9erwpdDOjH4N41I/u4HOiemMG3Vbn+3JUVMoSEiPitdKpTfdruMqYM7cElkaR5//1sefXcZe49oAGJJodAQkQvW7NJIPnu8A093b8qcDXvpnJjC5LQdGoBYAig0RCRfwkJDeOzahkwf2omml1TkqSkrue+NJew4oAGIwUyhISIXpUFUeSYObMtfejfj220H6fZCKm8u2MJZDUAMSgoNEbloISHGve2imZkQT+v6VXj287Xc/uo3pO/9yd+tSQFTaIhIgalVqQxv3n8NSXdeyeZ9x+g5ej4vz/leAxCDiEJDRAqUmXFLy9rMToinS7Ma/GvmRm56aT6rdh72d2tSABQaIlIoqpWPYMxdrRh379UcOHaKm8cu4O9faQBioFNoiEih6tbsEmYlxNOnVW1eTdlEj9HzWLx5v7/bknxSaIhIoYssU4pRfVrw/kNtOHPuHHeOX8QfPlvFTydO+7s1uUAKDREpMh0aVWPGsDgGdKzP+4u30y0plbnr9/q7LbkACg0RKVJlw8P4440xfPxYe8pFhPHAW0sZPuk7DhzTAMRAoNAQEb9oVbcyXwzpyJDrG/P5ih/okpjCFyt/0CiSYk6hISJ+ExEWSkKXJnz+REdqVS7D4A+WM/DdZezRAMRiS6EhIn53ec2KfPJYe/6nZ1NSN2bQOTGFiUu2611HMaTQEJFiISw0hIFxDZkxLI6YmhUZ8ckq7n59Mdv3awBicaLQEJFiJbpaOT58uC1/veUKVu48TNcXUnh93mYNQCwmFBoiUuyEhBh3tanLrIQ42jesxvNfruO2V75h4x4NQPQ3hYaIFFs1I8swoX8so/texfYDx7nhxXmMnv09p85oAKK/KDREpFgzM3pfVYtZw+Po0bwmSbM30uvl+azYccjfrZVICg0RCQhVy0fwYr+WvH5fLIeOn+aWsQv467R1/HxKAxCLkkJDRAJK55gazEyIo2/ruoxP3Uz30aks3KQBiEVFoSEiAadi6VL89ZYr+ODhNgD0e20RIz9ZxRENQCx0Cg0RCVjtG1Zj+tA4BsY1YNLS7XRNTOXrdXv83VZQ8yk0zKy7mW0ws3QzG5HD9ggzm+RtX2xm0Vm2jfTWN5hZt7xqmll9r0a6VzPcW08ws7VmttLMvjazehdz4iISHMqEh/I/PS/nk8c7EFmmFAPeTmPIh8vZf/Skv1sLSnmGhpmFAmOAHkAM0M/MYrLtNgA46JxrBCQBo7xjY4C+QDOgOzDWzELzqDkKSPJqHfRqAywHYp1zLYApwD/yd8oiEoyuqlOJz5/oyPDOTfhq9W66JKUy9btdGkVSwHx5p9EaSHfObXbOnQImAr2z7dMbeNt7PAW43szMW5/onDvpnNsCpHv1cqzpHXOdVwOv5s0Azrm5zrlf5gksAmpf+OmKSDALDwthaOfGfDmkE3WrlGXoxO946O00dh/+2d+tBQ1fQqMWsCPL853eWo77OOfOAIeBquc5Nrf1qsAhr0ZurwWZ7z6+yqlZMxtoZmlmlpaRkZHnyYlI8GlSowIfP9aeP9xwOQs27aNLYirvL97GOY0iuWgBdyPczO4BYoF/5rTdOTfeORfrnIuNiooq2uZEpNgIDTEe6tSAmcPiaVE7kt9/upq7Xl/E1n3H/N1aQPMlNHYBdbI8r+2t5biPmYUBkcD+8xyb2/p+oJJX41evZWadgd8DvZxzusslInmqW7Us7z/Uhr/fegVrdh2h2wupjE/dxJmzGkWSH76ExlKgsfetpnAyb2wnZ9snGejvPe4DzHGZd5+Sgb7et6vqA42BJbnV9I6Z69XAqzkVwMxaAuPIDAz9UGER8ZmZ0bd1XWYlxNOpcRR/nbae2175hvU/HvF3awEnz9Dw7i8MBmYA64DJzrk1ZvacmfXydpsAVDWzdCABGOEduwaYDKwFpgODnHNnc6vp1XoaSPBqVfVqQ+bHUeWBj8zsOzPLHlwiIud1SWRpXrvval6+qyU7D/7MjS/OJ3HWRk6e0SgSX1kwfx0tNjbWpaWl+bsNESmGDh47xXNfrOXT5btoUqM8o25rQcu6lf3dVrFgZsucc7E5bQu4G+EiIgWhcrlwku68ijfvv4afTpzh1le+4S9frOX4qTN5H1yCKTREpET7TdPqzBwex91t6jJh/ha6vZDKgvR9/m6r2FJoiEiJV6F0KZ6/+QomDWxLWEgId7++mBEfr+TwzxqAmJ1CQ0TE06ZBVb4a2olH4hswOW0HXRJTmLnmR3+3VawoNEREsihdKpSRPS7ns0EdqFIunIHvLmPwB9+yTwMQAYWGiEiOWtTOHID4265NmLlmD50TU/h0+c4SPwBRoSEikotSoSEMvq4x04Z2pEG1cgyftIIH3lrKrkMldwCiQkNEJA+Nqlfgo0fb8+ebYli8+QBdE1N4d1HJHICo0BAR8UFoiPFAh/rMHB5Hy7qV+eNnq+k7fhGbM476u7UipdAQEbkAdaqU5d0BrflHnxas//EIPUbP49WUkjMAUaEhInKBzIw7YuswOyGeay+L4u9frefmsQtY+0PwD0BUaIiI5FP1iqUZd28sr9zdih8Pn6TXy/P514wNnDgdvAMQFRoiIhepxxU1mZ0QR++ravHy3Lv0pCQAAAjsSURBVHRueHEey7Yd8HdbhUKhISJSACqVDeffd1zJ2w+25sTpc/R5dSHPJK/h2MngGoCo0BARKUDxTaKYMTyO+9rW461vttLthVTmfZ/h77YKjEJDRKSAlY8I49nezfno0XaEh4Vw74Ql/O6jFRw+HvgDEBUaIiKF5JroKkwb0onHr23IJ8t30Tkphemrd/u7rYui0BARKUSlS4XyVPemTB3UgajyETz63rc89t4y9v50wt+t5YtCQ0SkCDSvFcnUwR34XbfL+Hr9XrokpjJlWeANQFRoiIgUkVKhIQz6TSOmDelE4+rl+e1HK+j/5lJ2Hjzu79Z8ptAQESlijaqXZ/Ij7Xi2VzPSth6ga1Iqb3+zNSAGICo0RET8ICTE6N8+mpnD44iNrsKfk9dwx7iFpO8t3gMQFRoiIn5Uu3JZ3n7gGv59+5V8v/coPUfPY8zcdE4X0wGICg0RET8zM267ujazE+LpHFOdf87YQO+XF7B612F/t/YrCg0RkWIiqkIEY+++mlfvaUXG0ZP0HrOAUdPXF6sBiAoNEZFipnvzmsweHs+tLWvxyn820XP0PJZuLR4DEBUaIiLFUGTZUvzz9it5d0BrTp09x+2vLuRPU1dz1M8DEBUaIiLFWKfGUcwYFscDHaJ5d9E2uiWl8p8Ne/3Wj0JDRKSYKxcRxp9vasaUR9tTJjyU+99cSsLk7zh47FSR96LQEBEJEFfXq8yXQzryxHWNSP7uB7okpTBt1e4iHUWi0BARCSARYaE82fUykgd3pGZkGR5//1sefW8Ze48UzQBEn0LDzLqb2QYzSzezETlsjzCzSd72xWYWnWXbSG99g5l1y6ummdX3aqR7NcPzeg0RkZIm5tKKfPp4e0b0aMp/NmTQOTGFyWk7Cv1dR56hYWahwBigBxAD9DOzmGy7DQAOOucaAUnAKO/YGKAv0AzoDow1s9A8ao4CkrxaB73aub6GiEhJFRYawqPxDflqaCea1qzIU1NWcu+EJew4UHgDEH15p9EaSHfObXbOnQImAr2z7dMbeNt7PAW43szMW5/onDvpnNsCpHv1cqzpHXOdVwOv5s15vIaISInWIKo8Ex9uy/M3N+e7HYfompTK5yt+KJTX8iU0agE7sjzf6a3luI9z7gxwGKh6nmNzW68KHPJqZH+t3F7jv5jZQDNLM7O0jIzg+bm8IiLnExJi3NO2HjOHx9GhUTXqVytXOK9TKFX9yDk33jkX65yLjYqK8nc7IiJF6tJKZXi9fyzNa0UWSn1fQmMXUCfL89reWo77mFkYEAnsP8+xua3vByp5NbK/Vm6vISIiRcSX0FgKNPa+1RRO5o3t5Gz7JAP9vcd9gDku8xZ+MtDX++ZTfaAxsCS3mt4xc70aeDWn5vEaIiJSRMLy2sE5d8bMBgMzgFDgDefcGjN7DkhzziUDE4B3zSwdOEBmCODtNxlYC5wBBjnnzgLkVNN7yaeBiWb2PLDcq01uryEiIkXHgvkv67GxsS4tLc3fbYiIBBQzW+aci81pW9DdCBcRkcKj0BAREZ8pNERExGcKDRER8VlQ3wg3swxgWz4PrwbsK8B2AoHOuWTQOZcMF3PO9ZxzOf7r6KAOjYthZmm5fXsgWOmcSwadc8lQWOesj6dERMRnCg0REfGZQiN34/3dgB/onEsGnXPJUCjnrHsaIiLiM73TEBERnyk0RETEZwqNHJhZdzPbYGbpZjbC3/0UFDOrY2ZzzWytma0xs6HeehUzm2Vm33v/W9lbNzN70ft9WGlmrfx7Bvnj/Vz65Wb2hfe8vpkt9s5rkjeeH2+E/yRvfbGZRfuz7/wys0pmNsXM1pvZOjNrVwKu8XDvz/RqM/vQzEoH43U2szfMbK+Zrc6ydsHX1sz6e/t/b2b9c3qt3Cg0sjGzUGAM0AOIAfqZWYx/uyowZ4AnnXMxQFtgkHduI4CvnXONga+955D5e9DY+zUQeKXoWy4QQ4F1WZ6PApKcc42Ag8AAb30AcNBbT/L2C0SjgenOuabAlWSee9BeYzOrBQwBYp1zzcn8cQt9Cc7r/BbQPdvaBV1bM6sC/BloA7QG/vxL0PjEOadfWX4B7YAZWZ6PBEb6u69COtepQBdgA1DTW6sJbPAejwP6Zdn/f/cLlF9k/vTHr4HrgC8AI/NfyYZlv95k/nyXdt7jMG8/8/c5XOD5RgJbsvcd5Ne4FrADqOJdty+AbsF6nYFoYHV+ry3QDxiXZf2/9svrl95p/NovfwB/sdNbCyreW/KWwGKghnNut7fpR6CG9zgYfi9eAJ4CznnPqwKHnHNnvOdZz+l/z9fbftjbP5DUBzKAN72P5F43s3IE8TV2zu0C/gVsB3aTed2WEdzXOasLvbYXdc0VGiWQmZUHPgaGOeeOZN3mMv/qERTfwzazG4G9zrll/u6lCIUBrYBXnHMtgWP838cVQHBdYwDvo5XeZAbmpUA5fv0RTolQFNdWofFru4A6WZ7X9taCgpmVIjMw3nfOfeIt7zGzmt72msBebz3Qfy86AL3MbCswkcyPqEYDlczslx91nPWc/vd8ve2RwP6ibLgA7AR2OucWe8+nkBkiwXqNAToDW5xzGc6508AnZF77YL7OWV3otb2oa67Q+LWlQGPvmxfhZN5QS/ZzTwXCzIzMn7W+zjmXmGVTMvDLNyj6k3mv45f1+7xvYbQFDmd5G1zsOedGOudqO+eiybyOc5xzdwNzgT7ebtnP95ffhz7e/gH1N3Ln3I/ADjO7zFu6HlhLkF5jz3agrZmV9f6M/3LOQXuds7nQazsD6Gpmlb13aV29Nd/4+6ZOcfwF9AQ2ApuA3/u7nwI8r45kvnVdCXzn/epJ5ue5XwPfA7OBKt7+RuY3yTYBq8j8dorfzyOf534t8IX3uAGwBEgHPgIivPXS3vN0b3sDf/edz3O9CkjzrvNnQOVgv8bAs8B6YDXwLhARjNcZ+JDM+zanyXxXOSA/1xZ40Dv/dOCBC+lBY0RERMRn+nhKRER8ptAQERGfKTRERMRnCg0REfGZQkNERHym0BAREZ8pNERExGf/H122+6ge95q1AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UT9QLSjNdDRP"
      },
      "source": [
        "from torch.optim.lr_scheduler import LambdaLR\n",
        "learning_rate = 1e-4\n",
        "optimizer = AdamW(model.parameters(),lr=learning_rate)\n",
        "lambda1 = lambda epoch: epoch // 30\n",
        "\n",
        "scheduler = LambdaLR(optimizer, lr_lambda, last_epoch)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-cysCpTnbUtP"
      },
      "source": [
        "scheduler = get_cosine_schedule_with_warmup(optimizer, warmup_steps, total_steps)\n",
        "import math\n",
        "\n",
        "import torch\n",
        "from torch.optim import Optimizer\n",
        "from torch.optim.lr_scheduler import LambdaLR\n",
        "\n",
        "\n",
        "def get_cosine_schedule_with_warmup(\n",
        "  optimizer: Optimizer,\n",
        "  num_warmup_steps: int,\n",
        "  num_training_steps: int,\n",
        "  num_cycles: float = 0.5,\n",
        "  last_epoch: int = -1,\n",
        "):\n",
        "  \"\"\"\n",
        "  Create a schedule with a learning rate that decreases following the values of the cosine function between the\n",
        "  initial lr set in the optimizer to 0, after a warmup period during which it increases linearly between 0 and the\n",
        "  initial lr set in the optimizer. Args:\n",
        "    optimizer (:class:`~torch.optim.Optimizer`):\n",
        "      The optimizer for which to schedule the learning rate.\n",
        "    num_warmup_steps (:obj:`int`):\n",
        "      The number of steps for the warmup phase.\n",
        "    num_training_steps (:obj:`int`):\n",
        "      The total number of training steps.\n",
        "    num_cycles (:obj:`float`, `optional`, defaults to 0.5):\n",
        "      The number of waves in the cosine schedule (the defaults is to just decrease from the max value to 0\n",
        "      following a half-cosine).\n",
        "    last_epoch (:obj:`int`, `optional`, defaults to -1):\n",
        "      The index of the last epoch when resuming training.\n",
        "\n",
        "  Return:\n",
        "    :obj:`torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule.\n",
        "  \"\"\"\n",
        "def lr_lambda(current_step):\n",
        "    # Warmup\n",
        "    if current_step < num_warmup_steps:\n",
        "      return float(current_step) / float(max(1, num_warmup_steps))\n",
        "    # decadence\n",
        "    progress = float(current_step - num_warmup_steps) / float(\n",
        "      max(1, num_training_steps - num_warmup_steps)\n",
        "    )\n",
        "    return max(\n",
        "      0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))\n",
        "    )\n",
        "\n",
        "  return LambdaLR(optimizer, lr_lambda, last_epoch)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WWvFMzmqirLs",
        "outputId": "c71f3776-7048-4618-b20e-8e2ec4923391"
      },
      "source": [
        "import random\n",
        "a = random.randint(3,45)\n",
        "print(a)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "33\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}